# -*- coding: utf-8 -*-
import scrapy
import bs4

from lessons.baidu.baidu.items import UserItem


class NewsSpider(scrapy.Spider):
    name = 'news'
    allowed_domains = ['10.2.0.193']
    start_urls = ['http://10.2.0.193:8000/exercises/spider/level1/']
    #
    # def start_requests(self):
    #     pass

    def parse(self, response):
        bs = bs4.BeautifulSoup(response.text, features="lxml")
        for user_info in bs.find_all("tr"):
            assert isinstance(user_info, bs4.element.Tag)
            username, age, gender = list(user_info.children)
            item = UserItem()
            item["name"] = username.text
            item["age"] = int(age.text)
            item["gender"] = gender.text
            yield item

